package com.itheima.jdspider;

import com.itheima.jdspider.pojo.Item;
import org.apache.http.HttpEntity;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.sql.SQLException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.TimeUnit;

public class JDSpiderDemo2 {
    public static void main(String[] args) throws Exception {

        CloseableHttpClient client= HttpClients.createDefault();

        ArrayBlockingQueue<String> urls = new ArrayBlockingQueue<String>(100);
        for (int page = 1; page < 100; page++) {
            urls.offer("https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&wq=%E6%89%8B%E6%9C%BA&pvid=79a99cbd961f43afa63d35d0785a5211&page="+(page*2-1)+"&click=0");
        }

        //调度线程池执行器
        ScheduledThreadPoolExecutor scheduledThreadPoolExecutor = new ScheduledThreadPoolExecutor(6);

        //加入一条监控线程，如果容器中的数据没有了，就会停止
        scheduledThreadPoolExecutor.scheduleAtFixedRate(new Runnable() {
            @Override
            public void run() {
                int size = urls.size();
                if(size==0){

                    try {
                        scheduledThreadPoolExecutor.shutdown();
                        client.close();
                    } catch (IOException e) {
                        e.printStackTrace();
                    }
                }
            }
        },1,15,TimeUnit.SECONDS);

        /*
         * ScheduledFuture<?> scheduleWithFixedDelay(Runnable command, long initialDelay, long delay, TimeUnit unit)
         * 创建并执行在给定的初始延迟之后首先启用的定期动作，随后在一个执行的终止和下一个执行的开始之间给定的延迟。
         */
        for (int i = 0; i < 5; i++) {
            scheduledThreadPoolExecutor.scheduleWithFixedDelay(new Runnable() {
                @Override
                public void run() {
                    try {
                        String indexUrl = urls.poll();
                        if(indexUrl==null){
                            return;
                        }

                        //线程要执行任务
                        String html = getHtml(indexUrl, client);
                        //解析html
                        if(html!=null){
                            List<Item> itemList = getItemList(html, client);
                            System.out.println("收到"+itemList.size()+"个数据");
                            //保存数据
                            saveDao(itemList);
                        }
                    }catch (Exception e){
                        e.printStackTrace();
                    }
                }
            }, 0, 10, TimeUnit.SECONDS);
        }

        //client.close();
    }

    public static String getHtml(String indexUrl, CloseableHttpClient client) throws IOException {
        HttpGet httpGet = new HttpGet(indexUrl);

        httpGet.setHeader("user-agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36");
        CloseableHttpResponse resp = client.execute(httpGet);

        if(resp.getStatusLine().getStatusCode()==200){
            HttpEntity entity = resp.getEntity();
            String html = EntityUtils.toString(entity);
            return html;
        }
        resp.close();
        return null;
    }

    public static List<Item> getItemList(String html,CloseableHttpClient client) throws IOException {
        //解析数据
        Document document = Jsoup.parse(html);

        //获取所有商品的li标签
        Elements lis = document.select("#J_goodsList>[class=gl-warp clearfix]>li");
        ArrayList<Item> items = new ArrayList<>();
        for (Element li : lis) {
            //获取每个li的图片

            Elements imgs = li.select(".gl-i-wrap>.p-img>a>img");
            String imgUrl = "https:" + imgs.attr("data-lazy-img");

            //根据图片地址发送请求获取字节流
            HttpGet imgGet = new HttpGet(imgUrl);
            CloseableHttpResponse imgResp = client.execute(imgGet);

            HttpEntity entity1 = imgResp.getEntity();
            InputStream imgInput = entity1.getContent();
            String imgFileName = "C:\\Users\\86186\\Desktop\\jd\\" + UUID.randomUUID().toString() + imgUrl.substring(imgUrl.lastIndexOf("."));
            FileOutputStream outputStream = new FileOutputStream(imgFileName);
            byte[] bytes = new byte[1024];
            int len;
            while ((len = imgInput.read(bytes)) != -1) {
                outputStream.write(bytes, 0, len);
            }

            outputStream.close();
            imgInput.close();
            imgResp.close();

            String sku = li.attr("data-sku");
            String spu = li.attr("data-spu");
            if (spu == null || "".equals(spu)) {
                spu = sku;
            }

            //商品名称
            Elements name = li.select(".gl-i-wrap>div[class=p-name p-name-type-2]>a>em");
            String title = name.text();
            //商品价格

            Elements eleprice = li.select(".gl-i-wrap>.p-price>strong>i");
            String price = eleprice.text();


            //商品的url
            Elements p_url = li.select(".gl-i-wrap>.p-img>a");
            String url = "https:" + p_url.attr("href");

            //封装数据
            SimpleDateFormat daf = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss");

            Item item = new Item(null,
                    Long.parseLong(spu),
                    Long.parseLong(sku),
                    title, Double.parseDouble(price),
                    imgFileName, url,
                    daf.format(new Date()),
                    daf.format(new Date()));
            //封装到集合中
            items.add(item);
        }
        return items;
    }

    public static void saveDao(List<Item> items) throws SQLException {
        JDItemDao jdItemDao = new JDItemDao();
        jdItemDao.saveItem(items);
    }
}
